package net.bwie.realtime.jtp.dwd.log.job;

import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.dwd.log.function.IsNewFunction2;
import net.bwie.realtime.jtp.dwd.log.function.SplitFunction2;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class JtpAppLogLx2 {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStream<String> kafkaDataStream= KafkaUtil.consumerKafka(env, "topic-log");

        DataStream<String> pageStream = processLog(kafkaDataStream);

        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-logs");

        env.execute("JtpAppLogLx2");
    }

    private static DataStream<String> processLog(DataStream<String> stream) {
        //1.数据清洗
        DataStream<String> jsonStream = cleanedStream(stream);
        //2.新老访客状态标记修复
        DataStream<String> etlStream = isNewStream(jsonStream);
        //3.数据分流
        DataStream<String> pageStream = splitStream(etlStream);

        return pageStream;
    }

    private static DataStream<String> splitStream(DataStream<String> stream) {
        final OutputTag<String> errorTag = new OutputTag<String>("error-log"){};
        final OutputTag<String> startTag = new OutputTag<String>("start-log"){};
        final OutputTag<String> displayTag = new OutputTag<String>("display-log"){};
        final OutputTag<String> actionTag = new OutputTag<String>("action-log"){};

        SingleOutputStreamOperator<String> pageTagStream = stream.process(new SplitFunction2(errorTag, startTag, displayTag, actionTag));

        DataStream<String> errorStream = pageTagStream.getSideOutput(errorTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-logs");
        DataStream<String> startStream = pageTagStream.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-logs");
        DataStream<String> displayStream = pageTagStream.getSideOutput(displayTag);
        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-logs");
        DataStream<String> actionStream = pageTagStream.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-logs");

        return pageTagStream;
    }

    private static DataStream<String> isNewStream(DataStream<String> stream) {
        KeyedStream<String, String> midStream = stream.keyBy(new KeySelector<String, String>() {
            @Override
            public String getKey(String s) throws Exception {
                return JSON.parseObject(s).getJSONObject("common").getString("mid");
            }
        });
        DataStream<String> midIsNewStream = midStream.process(new IsNewFunction2());

        return midIsNewStream;
    }

    private static DataStream<String> cleanedStream(DataStream<String> stream) {
        final OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log"){};

        SingleOutputStreamOperator<String> jsonStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
                try {
                    //解析json数据
                    JSON.parseObject(value);
                    //解析成功,主流输出
                    out.collect(value);
                } catch (Exception e) {
                    ctx.output(dirtyTag, value);
                }
            }
        });

        SideOutputDataStream<String> dirtyStream = jsonStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-dirty-logs");
        return jsonStream;
    }
}
